package com.example.demo.crawlscript;

import cn.edu.hfut.dmic.contentextractor.ContentExtractor;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.rocks.BreadthCrawler;
import com.example.demo.es.NewsEntity;
import com.example.demo.es.TypeEnums;
import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.List;

public class CSDNCrawler extends BreadthCrawler {
    private List<NewsEntity> newsEntities = new ArrayList<>();
    public CSDNCrawler(String crawlPath, boolean autoParse) {
        super(crawlPath, autoParse);
        this.addSeed("https://blog.csdn.net/nav/java");
        this.addRegex("https://blog.csdn.net/.*/article/details/.*");
        this.addRegex("-.*\\.(jpg|png|gif).*");
    }

    @Override
    public void visit(Page page, CrawlDatums crawlDatums) {
        String url = page.url();
        String contentByUrl = null;
        try {
            contentByUrl = ContentExtractor.getContentByUrl(url);
        } catch (Exception e) {
        }
        if (page.matchUrl("https://blog.csdn.net/.*/article/details/.*")){
            String title = page.select("[id=articleContentId]").first().text();
            String authorImage = page.select("img[class=avatar_pic]").attr("src");
            String author = page.select("[id=uid]").first().text();//获取作者名
            if (!StringUtils.isAnyBlank(title, author, authorImage, contentByUrl)){
                NewsEntity newsEntity = new NewsEntity();
                newsEntity.setId(System.currentTimeMillis()+"");
                newsEntity.setTitle(title);
                newsEntity.setAuthor(author);
                newsEntity.setType(TypeEnums.WEB.getType());
                newsEntity.setAuthorImage(authorImage);
                newsEntity.setContent(contentByUrl);
                newsEntity.setUrl(page.url());
                newsEntities.add(newsEntity);
            }
        }
    }

    public static void main(String[] args) {
        CSDNCrawler csdnCrawler = new CSDNCrawler("test", true);
        try {
            csdnCrawler.start(2);
        } catch (Exception e) {

        }
    }

    public List<NewsEntity> getNewsEntities() {
        return newsEntities;
    }
}
